/*
 * Copyright (C) 2011 TopCoder Inc., All Rights Reserved.
 */
package gov.nasa.pds.services.impl;

import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.LineNumberReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Scanner;
import java.util.Set;
import java.util.regex.MatchResult;

import org.springframework.beans.factory.InitializingBean;

import com.topcoder.commons.utils.LoggingWrapperUtility;
import com.topcoder.commons.utils.ValidationUtility;
import com.topcoder.util.log.Level;
import com.topcoder.util.log.Log;

import gov.nasa.pds.entities.MetadataFile;
import gov.nasa.pds.entities.MetadataObject;
import gov.nasa.pds.entities.Property;
import gov.nasa.pds.entities.SequenceValueProperty;
import gov.nasa.pds.services.DataSetProcessingConfigurationException;
import gov.nasa.pds.services.DataSetProcessingException;
import gov.nasa.pds.services.MetadataFileReader;

/**
 * <p>
 * The <code>MetadataFileReaderImpl</code> class implements <code>MetadataFileReader</code>
 * interface by providing implementation of <code>readMetadataInfo(String)</code> method.
 * </p>
 *
 * <strong>Thread Safety:</strong> This class is mutable since it provides public setter for the logger. But it
 * doesn't change its state and is thread safe when the following conditions are met: this class is initialized by
 * Spring right after construction and its parameters are never changed after that, all entities passed to this class
 * are used by the caller in thread safe manner (accessed from a single thread only).
 *
 * @author KennyAlive
 * @version 1.0
 */
public class MetadataFileReaderImpl implements MetadataFileReader, InitializingBean {
    /**
     * Constant for the class name of this class. Used for logging.
     */
    private static final String CLASS_NAME = MetadataFileReaderImpl.class.getName();

    /**
     * Constant for line separator
     */
    private static final String LINE_SEPARATOR = System.getProperty("line.separator");

    /**
     * Ignore all objects and properties definitions which contain this word.
     */
    private static final String IGNORE_PATTERN = "SFDU";

    /**
     * Constant for OBJECT keyword.
     */
    private static final String OBJECT_KEYWORD = "OBJECT";


    // ---- Keywords and suffixes which indicates that whitespace should be preserved ----
    /**
     * Constant for _NOTE suffix.
     */
    private static final String WHITESPACE_PRESERVE_NOTE_SUFFIX = "_NOTE";
    /**
     * Constant for _DESC suffix.
     */
    private static final String WHITESPACE_PRESERVE_DESC_SUFFIX = "_DESC";
    /**
     * Constant for _DESCRIPTION suffix.
     */
    private static final String WHITESPACE_PRESERVE_DESCRIPTION_SUFFIX = "_DESCRIPTION";
    /**
     * Constant for NOTE keyword.
     */
    private static final String WHITESPACE_PRESERVE_NOTE_KEYWORD = "NOTE";
    /**
     * Constant for DESCRIPTION keyword.
     */
    private static final String WHITESPACE_PRESERVE_DESCRIPTION_KEYWORD = "DESCRIPTION";

    // --- Prefix should be ignored for object names that end with these suffixes -----
    /**
     * Constant for _TABLE suffix
     */
    private static final String IGNORE_PREFIX_INDICATOR_TABLE = "_TABLE";
    /**
     * Constant for _SERIES suffix
     */
    private static final String IGNORE_PREFIX_INDICATOR_SERIES = "_SERIES";
    /**
     * Constant for _LABEL suffix
     */
    private static final String IGNORE_PREFIX_INDICATOR_LABEL = "_LABEL";
    /**
     * Constant for _TEXT suffix
     */
    private static final String IGNORE_PREFIX_INDICATOR_TEXT = "_TEXT";
    /**
     * Constant for _HEADER suffix
     */
    private static final String IGNORE_PREFIX_INDICATOR_HEADER = "_HEADER";
    /**
     * Constant for _IMAGE suffix
     */
    private static final String IGNORE_PREFIX_INDICATOR_IMAGE = "_IMAGE";


    /**
     * This set contains pointer names which refer to files that should be included
     * while parsing the main file.
     */
    private static final Set<String> INCLUDE_POINTERS_NAMES;

    /**
     * Initializes <code>INCLUDE_POINTERS_NAMES</code> set.
     */
    static {
        Set<String> set = new HashSet<String>();
        set.add("^STRUCTURE");
        set.add("^CATALOG");
        set.add("^DATA_SET_MAP_PROJECTION");
        INCLUDE_POINTERS_NAMES = Collections.unmodifiableSet(set);
    }

    /**
     * <p>
     * The <code>ParserContext</code> class represents a state of the metadata file parser.
     * The metadata parser itself is thread safe. This is mostly achieved by the recursive implementation of the
     * parser. All the necessary info are passed through the methods parameters. The <code>ParserContext</code>
     * class holds the parser state that is used my the most methods of the implementation, thus reducing the number
     * of the methods' parameters and grouping parser state variables in one place.
     * </p>
     *
     * <strong>Thread Safety:</strong> This class is not thread safe since it has mutable fields.
     *
     * @author TCSASSEMBLER
     * @version 1.0
     */
    private static class ParserContext {
        /**
         * Represents the filename of the file to parse.
         */
        private final String filename;

        /**
         * The reader for reading the next line from the file.
         */
        private LineNumberReader reader;

        /**
         * Represents the line that is being processed. Can be <code>null</code>. Has getter and setter.
         */
        private String currentLine;

        /**
         * Represents the next line that should be parser. Can be <code>null</code>. Has getter and setter.
         */
        private String nextLine;

        /**
         * Creates an instance of <code>ParserContext</code>.
         *
         * @param filename
         *              the filename of the file to parse
         *
         * @throws FileNotFoundException
         *              if there is an error while opening the file
         */
        public ParserContext(String filename) throws FileNotFoundException {
            this.filename = filename;
            this.reader = new LineNumberReader(new FileReader(filename));
        }

        /**
         * Gets the filename of the file to parse
         *
         * @return the filename of the file to parse
         */
        public String getFileName() {
            return filename;
        }

        /**
         * Gets the reader associated with this parser context.
         *
         * @return the buffered reader to read text for parsing
         */
        public BufferedReader getReader() {
            return reader;
        }

        /**
         * Gets the current line number of the file being parsed.
         *
         * @return the current line number of the file being parsed
         */
        public int getCurrentLineNumber() {
            return reader.getLineNumber();
        }

        /**
         * Gets the current line.
         *
         * @return the line that is being processed
         */
        public String getCurrentLine() {
            return currentLine;
        }

        /**
         * Sets the current line.
         *
         * @param currentLine
         *              the current line to set
         */
        public void setCurrentLine(String currentLine) {
            this.currentLine = currentLine;
        }

        /**
         * Gets the next.
         *
         * @return the next line that should be processed or <code>null</code> if the next line is not set
         */
        public String getNextLine() {
            return nextLine;
        }

        /**
         * Sets the next line.
         *
         * @param nextLine
         *              the next line that should be processed (not null)
         */
        public void setNextLine(String nextLine) {
            this.nextLine = nextLine;
        }
    }

    /**
     * <p>
     * The <code>Log</code> instance used for logging.
     * </p>
     *
     * <p>
     * It is initialized with Spring setter dependency injection. Cannot be <code>null</code> after initialization,
     * assuming that property is initialized via Spring setter-based dependency injection and is never changed
     * after that. Has a setter.
     * </p>
     */
    private Log logger;

    /**
     * Sets the logger.
     *
     * @param logger
     *              the logger to set
     */
    public void setLogger(Log logger) {
        this.logger = logger;
    }

    /**
     * Creates an instance of <code>MetadataFileReaderImpl</code>.
     */
    public MetadataFileReaderImpl() {
        // Scanner#hasNextDouble() method depends on the locale settings to determine how the decimal separator
        // looks like. We set ENGLISH locale to guarantee that the dot separator is '.'
        Locale.setDefault(Locale.ENGLISH);
    }

    /**
     * Checks whether this class was initialized by Spring properly.
     *
     * Required parameters:
     * <ul>
     * <li>logger</li>
     * </ul>
     *
     * @throws DataSetProcessingConfigurationException
     *            if the class was not initialized properly
     */
    @Override
    public void afterPropertiesSet() {
        ValidationUtility.checkNotNull(logger, "logger", DataSetProcessingConfigurationException.class);
    }

    /**
     * Reads the metadata from the given file.
     *
     * @param filename
     *              the filename of the file with the metadata
     *
     * @return the parsed metadata file object
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    @Override
    public MetadataFile readMetadataInfo(String filename) throws DataSetProcessingException {
        String signature = CLASS_NAME + ".readMetadataInfo(String filename)";
        LoggingWrapperUtility.logEntrance(logger, signature, new String[] {"filename"}, new Object[] {filename});

        MetadataFile metadataFile = readMetadataInfoInternal(filename);

        LoggingWrapperUtility.logExit(logger, signature, new Object[] {metadataFile.toJSONString()});
        return metadataFile;
    }

    /**
     * Implementation of the metadata reading algorithm.
     *
     * @param filename
     *              the filename to read
     *
     * @return the metadata file object
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    private MetadataFile readMetadataInfoInternal(String filename) throws DataSetProcessingException {
        final String signature = CLASS_NAME + ".readMetadataInfoInternal(String filename)";

        // initialize parser context
        ParserContext context = null;
        try {
            context = new ParserContext(filename);
        } catch (FileNotFoundException e) {
            throw LoggingWrapperUtility.logException(logger, signature, new DataSetProcessingException(
                    "Failed to open file with metadata info: " + filename, e));
        }

        // prepare containers for parsed data
        List<MetadataObject> metadataObjects = new ArrayList<MetadataObject>();
        List<Property> properties = new ArrayList<Property>();

        // parse objects and properties definitions by reading input file line by line
        String line = nextLine(context);
        while (line != null) {
            String trimmedLine = line.trim(); // for quoted multi-line properties we keep a copy of untrimmed line
            // skip comments and top-level empty lines
            if (trimmedLine.isEmpty() || trimmedLine.startsWith("/*")) {
                line = nextLine(context);
                continue;
            }
            // parse nested objects and properties
            if (trimmedLine.equals("END")) {
                // just ignore END keyword
            } else if (trimmedLine.matches("OBJECT(\\s.*)?")) {
                context.setCurrentLine(trimmedLine.substring(OBJECT_KEYWORD.length()));
                MetadataObject metadataObject = parseMetadataObject(context);
                // parseMetadataObject returns null if the object is ignored (it contains a property with IGNORE_PATTERN)
                if (metadataObject != null) {
                    metadataObjects.add(metadataObject);
                }
            } else {
                context.setCurrentLine(line);
                Property property = parseProperty(context);
                // parseProperty returns null if the property is ignored (it contains IGNORE_PATTERN)
                if (property != null) {
                    // check if this property is an include property
                    if (INCLUDE_POINTERS_NAMES.contains(property.getName())) {
                        processIncludeProperty(property, metadataObjects, properties);
                    } else {
                        properties.add(property);
                    }
                }
            }
            line = nextLine(context);
        }

        // create MetadataFile instance and initialize it with collected data
        MetadataFile metadataFile = new MetadataFile(filename);
        metadataFile.setChildren(metadataObjects);
        metadataFile.setProperties(properties);
        return metadataFile;
    }

    /**
     * Parses object definition.
     *
     * @param context
     *              the parser context
     *
     * @return the metadata object
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    private MetadataObject parseMetadataObject(ParserContext context) throws DataSetProcessingException {
        // parse equals sign token and object name
        parseEqualSign(context);
        String objectName = parseObjectName(context);

        // prepare containers for parsed data
        List<MetadataObject> childObjects = new ArrayList<MetadataObject>();
        List<Property> childProperties = new ArrayList<Property>();

        boolean ignoreThisObject = false; // object is ignored when one of its properties is ignored

        // parse sub-objects/properties of the current object
        String line = nextLine(context);
        while (line != null) {
            String trimmedLine = line.trim(); // for quoted multi-line properties we keep a copy of untrimmed line
            // skip comments and empty lines
            if (trimmedLine.isEmpty() || trimmedLine.startsWith("/*")) {
                line = nextLine(context);
                continue;
            }
            // check for end-of-object mark
            if (trimmedLine.matches("END_OBJECT(\\s.*)?")) {
                validateEndOfObject(context, objectName);
                break;
            }
            // parse nested objects and properties
            if (trimmedLine.matches("OBJECT(\\s.*)?")) {
                context.setCurrentLine(trimmedLine.substring(OBJECT_KEYWORD.length()));
                MetadataObject childObject = parseMetadataObject(context);
                // parseMetadataObject returns null if the object is ignored (it contains a property with IGNORE_PATTERN)
                if (childObject != null) {
                    childObjects.add(childObject);
                }
            } else {
                context.setCurrentLine(line);
                Property childProperty = parseProperty(context);
                // parseProperty returns null if the property is ignored (it contains IGNORE_PATTER)
                if (childProperty != null) {
                    // check if this property is an include property
                    if (INCLUDE_POINTERS_NAMES.contains(childProperty.getName())) {
                        processIncludeProperty(childProperty, childObjects, childProperties);
                    } else {
                        childProperties.add(childProperty);
                    }
                } else {
                    ignoreThisObject = true;
                }
            }
            line = nextLine(context);
        }

        // skip optional prefix
        int underscoreIndex = objectName.lastIndexOf('_');
        if (underscoreIndex != -1) {
            if (objectName.endsWith(IGNORE_PREFIX_INDICATOR_TABLE)
                    || objectName.endsWith(IGNORE_PREFIX_INDICATOR_SERIES)
                    || objectName.endsWith(IGNORE_PREFIX_INDICATOR_LABEL)
                    || objectName.endsWith(IGNORE_PREFIX_INDICATOR_TEXT)
                    || objectName.endsWith(IGNORE_PREFIX_INDICATOR_HEADER)
                    || objectName.endsWith(IGNORE_PREFIX_INDICATOR_IMAGE)) {
                String oldName = objectName;
                objectName = objectName.substring(underscoreIndex + 1);
                logger.log(Level.DEBUG, "Skipping object name prefix. Original name {0}, new name {1}",
                        oldName, objectName);
            }
        }

        if (ignoreThisObject) {
            return null;
        }
        // create MetadataObject instance and initialize it with collected data
        MetadataObject metadataObject = new MetadataObject(objectName);
        metadataObject.setChildren(childObjects);
        metadataObject.setProperties(childProperties);
        return metadataObject;
    }

    /**
     * Checks that END_OBJECT definition is correct.
     *
     * @param context
     *              the parser context
     * @param objectName
     *              the name of the object which definition is finished by the END_OBJECT keyword
     *
     * @throws DataSetProcessingException
     *                  if END_OBJECT definition is malformed
     */
    private void validateEndOfObject(ParserContext context, String objectName) throws DataSetProcessingException {
        final String signature = CLASS_NAME + ".validateEndOfObject(ParserContext context)";

        Scanner lineScanner = new Scanner(context.getCurrentLine());
        lineScanner.next(); // skip END_OBJECT token

        boolean valid = true;
        boolean equalsFound = false;
        boolean objectNameFound = false;

        // process current line
        if (lineScanner.hasNext()) {
            String token = lineScanner.next();
            if (!token.equals("=")) {
                valid = false; // after END_OBJECT there is a token, but it's not a '='
            } else {
                equalsFound = true;
            }
        }

        // search for optional '=' token if it is not found yet
        if (valid && !equalsFound) {
            context.setCurrentLine(null); // this forces moveToNextToken to start with a new line
            moveToNextToken(context);
            lineScanner = new Scanner(context.getCurrentLine());
            String token = lineScanner.next();
            if (token.equals("=")) {
               equalsFound = true;
            } else {
                // it's important: if we don't need this line then returns it back
                context.setNextLine(context.getCurrentLine());
            }
        }

        // search for object name: it should be present of equals sign is already found
        if (valid && equalsFound && !objectNameFound) {
            if (!lineScanner.hasNext()) {
                context.setCurrentLine(null); // this forces moveToNextToken to start with a new line
                moveToNextToken(context);
                lineScanner = new Scanner(context.getCurrentLine());
            }
            if (!lineScanner.next().equals(objectName)) {
                valid = false; // token after END_OBJECT does not match object name
            }
            if (lineScanner.hasNext()) {
                valid = false; // there should be no more tokens after object name
            }
        }

        // throw an exception if END_OBJECT definition is malformed
        if (!valid) {
            throw parserException(context, signature, "Invalid END_OBJECT format");
        }
    }

    /**
     * Processes include property.
     *
     * @param includeProperty
     *              the include property to process
     * @param metadataObjects
     *              container for included metadata objects
     * @param properties
     *              container for included properties
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    private void processIncludeProperty(Property includeProperty, List<MetadataObject> metadataObjects,
        List<Property> properties) throws DataSetProcessingException {
        logger.log(Level.DEBUG, "Processing include property {0}, including file {1}",
                                includeProperty.getName(), includeProperty.getValues().get(0));
        MetadataFile metadataFile = readMetadataInfoInternal(includeProperty.getValues().get(0));
        metadataObjects.addAll(metadataFile.getChildren());
        properties.addAll(metadataFile.getProperties());
    }

    /**
     * Parses the property definition.
     *
     * @param context
     *              the parser context
     *
     * @return the property object
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    private Property parseProperty(ParserContext context) throws DataSetProcessingException {
        final String signature = CLASS_NAME + ".parseProperty(ParserContext context)";

        // get property name
        Scanner lineScanner = new Scanner(context.getCurrentLine());
        String propertyName = lineScanner.next();

        // parse equals sign token
        int equalsIndex = propertyName.indexOf("=");
        if (equalsIndex > 0) {
            String theRest = propertyName.substring(equalsIndex + 1);
            propertyName = propertyName.substring(0, equalsIndex);
            context.setCurrentLine(theRest + " " + lineScanner.nextLine());
        } else {
            context.setCurrentLine(lineScanner.nextLine());
            parseEqualSign(context);
        }

        // check whether we should preserve whitespace for this property's value
        boolean preserveWhitespace = false;
        if (propertyName.endsWith(WHITESPACE_PRESERVE_NOTE_SUFFIX)
                || propertyName.endsWith(WHITESPACE_PRESERVE_DESC_SUFFIX)
                || propertyName.endsWith(WHITESPACE_PRESERVE_DESCRIPTION_SUFFIX)
                || propertyName.equals(WHITESPACE_PRESERVE_NOTE_KEYWORD)
                || propertyName.equals(WHITESPACE_PRESERVE_DESCRIPTION_KEYWORD)) {
            preserveWhitespace = true;
        }

        // move to the beginning of property's value
        moveToNextToken(context);

        // parse various classes of property's values
        List<String> values = null;

        String line = context.getCurrentLine();
        lineScanner = new Scanner(context.getCurrentLine());
        if (line.matches("\\s*\\(.*")) { // it's either a sequence of sets or a set
            line = line.substring(line.indexOf('(') + 1); // skip the first '('
            boolean sequence = false;
            if (line.matches("\\s*\\(.*") || line.matches("\\s*\\{.*")) {
                sequence = true;
                context.setCurrentLine(line);
            } else if (line.matches("\\s*")) {
                context.setCurrentLine(line);
                moveToNextToken(context);
                if (!context.getCurrentLine().matches("\\s*\\(.*") && !context.getCurrentLine().matches("\\s*\\{.*")) {
                    throw parserException(context, signature, "Malformed sequence definition");
                }
                sequence = true;
            }

            if (sequence) {
                List<List<String>> sequences = parseSequence(context);
                SequenceValueProperty property = new SequenceValueProperty(propertyName);
                property.setSequences(sequences);
                return property;
            } else {
             // this parses set of words surrounded by ( )
                values = parseSet(context, false, false);
            }
        } else if (lineScanner.hasNext("\".*")) {
            // this parses single-line/multi-line quoted string or quoted word
            values = parseQuotedString(context, preserveWhitespace, false);
        } else if (lineScanner.hasNext("'.*")) {
            // this parses single-line/multi-line single quoted string or single quoted word
            values = parseQuotedString(context, preserveWhitespace, true);
        }
        else if (lineScanner.hasNext("\\{.*")) {
            // this parses set of words surrounded by { }
            values = parseSet(context, true, false);
        } else if (lineScanner.hasNext("\\(.*")) {
            // this parses set of words surrounded by ( )
            values = parseSet(context, false, false);
        } else {
            // this parses word (without quotes), number or number with unit
            values = parseSingleLineValue(context);
        }
        // ignore properties whose value contains ignore pattern
        if (values.size() == 1 && values.get(0).contains(IGNORE_PATTERN)) {
            logger.log(Level.DEBUG, "Ignoring property {0} = {1}", propertyName, values.get(0));
            return null;
        }
        Property property = new Property(propertyName);
        property.setValues(values);
        return property;
    }

    /**
     * Parses the single-line value.
     *
     * @param context
     *              the parser context
     *
     * @return the list with parsed string
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    private List<String> parseSingleLineValue(ParserContext context) throws DataSetProcessingException {
        final String signature = CLASS_NAME + ".parseSingleLineValue(Scanner lineScanner)";
        Scanner lineScanner = new Scanner(context.getCurrentLine());

        List<String> values = new ArrayList<String>();
        if (lineScanner.hasNextDouble()) {
            String number = lineScanner.next();
            if (lineScanner.hasNext() && !lineScanner.hasNext("/\\*.*")) {
                // get associated units
                String units = lineScanner.next();
                number += " " + units;
                if (!units.matches("<\\w+>")) {
                    logger.log(Level.WARN, "The units of the number are not surrounded by the angle brackes");
                }
            }
            values.add(number);
        } else {
            String word = lineScanner.next();
            values.add(word);
        }

        if (lineScanner.hasNext() && !lineScanner.nextLine().matches("\\s*/\\*.*")) {
            String message = "Invalid property value: unexpected token after property value";
            throw parserException(context, signature, message);
        }
        return values;
    }

    /**
    * Parses the quoted string.
    *
    * @param context
    *               the parser context
    * @param preserveWhitespace
    *               defines whether whitespaces should be preserved
    * @param singleQuotes
    *               defines whether a string is quoted with regular or single quotes
    *
    * @return the list with the parsed string
    *
    * @throws DataSetProcessingException
     *              if there is an error while parsing the file
    */
   private List<String> parseQuotedString(ParserContext context, boolean preserveWhitespace, boolean singleQuotes)
       throws DataSetProcessingException {
       final String signature = CLASS_NAME
               + ".parseQuotedString(ParserContext context, boolean preserveWhitespace,  boolean singleQuotes)";

       List<String> values = new ArrayList<String>();
       Scanner lineScanner = new Scanner(context.getCurrentLine());

       // check for single-line quoted string
       if (lineScanner.findInLine(singleQuotes ? "'([^\"]*)'" : "\"([^\"]*)\"") != null) {
           MatchResult result = lineScanner.match();
           String string =result.group(1);
           values.add(preserveWhitespace ? string : string.trim());
       } else { // multi-line quoted string
           lineScanner.findInLine(singleQuotes ? "'([^\\n\\r]*)" : "\"([^\\n\\r]*)");
           StringBuilder sb = new StringBuilder(lineScanner.match().group(1));

           while (true) {
               String line = nextRequiredLine(context);
               int quotePos = line.indexOf(singleQuotes ? '\'' : '\"');
               if (quotePos != -1) {
                   sb.append(LINE_SEPARATOR).append(line.substring(0, quotePos));
                   if (!line.substring(quotePos + 1).trim().isEmpty()) {
                       String message = "Invalid property definition: unexpected token was found after quoted string";
                       throw parserException(context, signature, message);
                   }
                   break;
               }
               sb.append(LINE_SEPARATOR).append(line);
           }
           values.add(preserveWhitespace ? sb.toString() : sb.toString().trim());
       }
       return values;
   }

    /**
     * Parses the set of words.
     *
     * @param context
     *               the parser context
     * @param curlyBrackets
     *               defines surrounding symbols: {} or ()
     *
     * @return the list of parsed words
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    private List<String> parseSet(ParserContext context, boolean curlyBrackets, boolean sequenceSet)
        throws DataSetProcessingException {
        final String signature = CLASS_NAME + ".parseSet(ParserContext context, boolean curlyBrackets)";

        List<String> values = new ArrayList<String>();

        char startSymbol = curlyBrackets ? '{' : '(';
        char endSymbol = curlyBrackets ? '}' : ')';

        String line = context.getCurrentLine();
        line = line.substring(line.indexOf(startSymbol) + 1); // start after opening symbol

        // retrieve the set of words
        while (true) {
            // skip comments and empty lines
            if (line.isEmpty() || line.startsWith("/*")) {
                line = nextRequiredLine(context).trim();
                continue;
            }

            String remainder = tokenizeUsingCommas(context, line, endSymbol, values);

            // check if set end symbol is found
            if (!remainder.isEmpty()) {
                remainder = remainder.substring(1).trim(); // skip end symbol
                if (!sequenceSet && !remainder.isEmpty()) {
                    String message = "Malformed set: unexpected token was found after set";
                    throw parserException(context, signature, message);
                }
                if (sequenceSet) {
                    context.setCurrentLine(remainder);
                }
                break;
            }
            // sets inside sequences should occupy single line
            if (sequenceSet) {
                String message = "Malformed set: the nested set object should be defined within a single line";
                throw parserException(context, signature, message);
            }
            line = nextRequiredLine(context).trim();
        }

        return values;
    }

    /**
     * Parses the sequence of sets of words.
     *
     * @param context
     *               the parser context
     *
     * @return the list of sets of words
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    private List<List<String>> parseSequence(ParserContext context) throws DataSetProcessingException {
        final String signature = CLASS_NAME + ".parseSequence(ParserContext context)";

        List<List<String>> sequence = new ArrayList<List<String>>();
        int setSize = -1; // stores the set size. All sets should have the same size

        String line = context.getCurrentLine();

        // Iterate over lines. This loop finishes when we find sequence close symbol ')' or when
        // the end of file is reached (in this case the exception will be thrown).
        while (true) {
            // skip comments and empty lines
            if (line.isEmpty() || line.startsWith("/*")) {
                line = nextRequiredLine(context).trim();
                continue;
            }
            // check the end of the sequence: at the beginning of the line
            if (line.startsWith(")")) {
                if (!line.substring(1).trim().isEmpty()) {
                    String message = "Malformed sequence: unexpected token is found after sequence definition";
                    throw parserException(context, signature, message);
                }
                break; // end of sequence of sets
            }

            // parse set definition.
            boolean curlyBrackets = line.matches("\\s*\\{.*");
            context.setCurrentLine(line);
            List<String> wordSet = parseSet(context, curlyBrackets, true);

            if (setSize == -1) {
                setSize = wordSet.size();
            } else if (wordSet.size() != setSize) {
                String message = "Malformed sequence: the sets have different sizes";
                throw parserException(context, signature, message);
            }
            sequence.add(wordSet);

            // check the end of the sequence: at the end of the line
            line = context.getCurrentLine();
            if (line.equals(")")) {
                break; // end of the sequence of sets
            } else if (!line.isEmpty() && !line.equals(",")) {
                String message = "Malformed sequence: unexpected token is found after set definition";
                throw parserException(context, signature, message);
            }
            line = nextRequiredLine(context).trim();
        }
        return sequence;
    }

    /**
     * Tokenizes the given string using comma as a delimiter. Does not take into account commas and braces that
     * are located between quotes.
     *
     * @param context
     *          the parser context
     * @param line
     *          not empty line of text from input file
     * @param endSymbol
     *          the end symbol for set: '}' or ')'
     * @param result
     *          container for result
     *
     * @return empty string if endSymbol is not found, otherwise the rest of the string starting from endSymbol
     */
    private String tokenizeUsingCommas(ParserContext context, String line, char endSymbol, List<String> result)
        throws DataSetProcessingException {
        final String signature = CLASS_NAME +
                ".tokenizeUsingCommas(ParserContext context, String line, char endSymbol, List<String> result)";

        boolean quote = false;
        char quoteChar = 0;

        if (line.charAt(0) == ',') {
            line = line.substring(1);
            if (line.isEmpty()) {
                return "";
            }
        }
        if (line.charAt(0) == '\'' || line.charAt(0) == '"') {
            quote = true;
            quoteChar = line.charAt(0);
        }

        int beginIndex = 0;
        int i = 1;
        for (; i < line.length(); i++) {
            char c = line.charAt(i);
            if (quote && c == quoteChar) {
                quote = false;
            } else if (!quote && (c == '\'' || c == '"')) {
                quote = true;
                quoteChar = c;
            }

            if (!quote) {
                if (c == ',') {
                    result.add(trimQuotesAndSpaces(line.substring(beginIndex, i)));
                    beginIndex = i + 1;
                    if (beginIndex < line.length() && line.charAt(i + 1) == ',') {
                        String message = "Malformed line: two commas in a row";
                        throw parserException(context, signature, message);
                    }
                } else if (c == endSymbol) {
                    break;
                }
            }
        }

        if (quote) {
            String message = "Malformed line: missing closed quote";
            throw parserException(context, signature, message);
        }
        if (beginIndex < line.length()) {
            result.add(trimQuotesAndSpaces(line.substring(beginIndex, i)));
        }
        return line.substring(i);
    }

    /**
     * Searches for equals sign token: in most cases it is on the same line but can also be on a separate line.
     *
     * @param context
     *              the parser context
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    private void parseEqualSign(ParserContext context) throws DataSetProcessingException {
        final String signature = CLASS_NAME + ".checkEqualSign(ParserContext context)";

        moveToNextToken(context);
        Scanner lineScanner = new Scanner(context.getCurrentLine());

        String token = lineScanner.next();
        if (!token.startsWith("=")) {
            // OBJECT or property name is not followed by '='
            String message = "Malformed definition: OBJECT or property name is not followed by '='";
            throw parserException(context, signature, message);
        }

        // initialize current line with with the rest of the line
        String line = lineScanner.hasNextLine() ? lineScanner.nextLine() : "";
        if (!token.equals("=")) {
            line = token.substring(1) + " " + line;
        }
        context.setCurrentLine(line);
    }

    /**
     * Searches for object name token: in most cases it is on the same line but can also be on a separate line.
     *
     * @param context
     *              the parser context
     *
     * @return the object name
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    private String parseObjectName(ParserContext context) throws DataSetProcessingException {
        final String signature = CLASS_NAME + ".checkObjectName(ParserContext context)";

        moveToNextToken(context);
        Scanner lineScanner = new Scanner(context.getCurrentLine());

        String objectName = lineScanner.next();
        // there should be no more tokens after the object name
        if (lineScanner.hasNext()) {
            String message = "Malformed object definition: unexpected token was found after the object name";
            throw parserException(context, signature, message);
        }
        context.setCurrentLine(null); // just to invalidate already processed line
        return objectName;
    }

    /**
     * Moves to the beginning of the next token by skipping empty lines (after this method returns it is allowed
     * to have whitespaces before the next token)
     *
     * @param context
     *              the parser context
     *
     * @throws DataSetProcessingException
     *              if there is an error while parsing the file
     */
    private void moveToNextToken(ParserContext context) throws DataSetProcessingException {
        String line = context.getCurrentLine();
        if (line == null) {
            line = "";
        }
        Scanner lineScanner = new Scanner(line);
        while (!lineScanner.hasNext()) {
            lineScanner = new Scanner(nextRequiredLine(context));
        }
        // initialize current line with with the rest of the line
        context.setCurrentLine(lineScanner.nextLine());
    }

    /**
     * Reads the next line from the input file. Returns <code>null</code> if the end of the stream has been reached.
     * The method also updates <code>ParserContext#currentLine</code>. If <code>ParserContext#nextLine</code> is
     * set then it is used as the current line. The <code>ParserContext#nextLine</code>is set to <code>null</code>
     * before this method returns.
     *
     * @param context
     *              the parser context
     *
     * @return the string with the content of the line or <code>null</code> if the end of the stream has been reached
     *
     * @throws DataSetProcessingException
     *                      if there is an error while reading the file
     */
    private String nextLine(ParserContext context) throws DataSetProcessingException {
        final String signature = CLASS_NAME + ".nextLine(ParserContext context)";
        try {
            String line = null;
            if (context.getNextLine() != null) {
                line = context.getNextLine();
                context.setNextLine(null);
            } else {
                line = context.getReader().readLine();
            }
            context.setCurrentLine(line);
            return line;
        } catch (IOException e) {
            String message = "Failed to read next line from file: " + context.getFileName();
            throw LoggingWrapperUtility.logException(logger, signature, new DataSetProcessingException(message));
        }
    }

    /**
     * Reads the next line from the input file. Throws exception if the end of stream has been reached.
     * The method also updates <code>ParserContext#currentLine</code>. If <code>ParserContext#nextLine</code> is
     * set then it is used as the current line. The <code>ParserContext#nextLine</code>is set to <code>null</code>
     * before this method returns.
     *
     * @param context
     *              the parser context
     *
     * @return the string with the content of the line (not null)
     *
     * @throws DataSetProcessingException
     *                      if there is an error while reading the file or the end of the file has been reached
     */
    private String nextRequiredLine(ParserContext context) throws DataSetProcessingException {
        final String signature = CLASS_NAME + ".nextRequiredLine(ParserContext context)";
        String line = nextLine(context);

        if (line == null) {
            String message = "Unexpected end of file: " + context.getFileName();
            throw LoggingWrapperUtility.logException(logger, signature, new DataSetProcessingException(message));
        }
        return line;
    }

    /**
     * Trims all surrounding whitespace symbols and quotes (both single and regular). It is assumed that
     * the method's <code>String</code> parameter is not <code>null</code>.
     *
     * @param str
     *              the string to trim surrounding whitespaces and quotes from
     *
     * @return the copy of the given string without quotes and leading/ending whitespace symbols
     */
    private String trimQuotesAndSpaces(String str) {
        int length = str.length();
        int prevLength = 0;
        while (length != prevLength) {
            str = str.trim();
            if (!str.isEmpty() && str.charAt(0) == '"') {
                str = str.substring(1);
            }
            if (!str.isEmpty() && str.charAt(str.length() - 1) == '"') {
                str = str.substring(0, str.length() - 1);
            }
            if (!str.isEmpty() && str.charAt(0) == '\'') {
                str = str.substring(1);
            }
            if (!str.isEmpty() && str.charAt(str.length() - 1) == '\'') {
                str = str.substring(0, str.length() - 1);
            }
            prevLength = length;
            length = str.length();
        }
        return str;
     }

    /**
     * Creates an instance of <code>DataSetProcessingException</code>. The provided error message is appended with
     * the file name that is parsed and current line number that approximately indicates where the malformed
     * construction is located. This methods also logs the exception before it returns.
     *
     * @param context
     *              the parser context
     * @param signature
     *              the method signature
     * @param message
     *              the error message
     *
     * @return the instance of <code>DataSetProcessingException</code>
     */
    private DataSetProcessingException parserException(ParserContext context, String signature, String message) {
        String errorMessage = String.format("%s [%s (%d)]", message, context.getFileName(),
                context.getCurrentLineNumber());
        return LoggingWrapperUtility.logException(logger, signature, new DataSetProcessingException(errorMessage));
    }
}
